import random


def load_data(path):
    """
    数据加载
    @param path: 文件路径
    @return: 句子列表
    """
    data = []
    file = open(path, 'r', encoding='utf8', errors='ignore')
    file_data = file.readlines()  # 读取所有行
    print(len(file_data))
    for row in file_data[1:]:
        if row == '\n' or len(row) == 0:
            continue
        row = row.replace('\n', '')

    data = file_data[1:].copy()
    print(len(data))
    random.shuffle(data)
    return data


def save_data(path, data):
    fileObject = open(path, 'w')
    fileObject.write('label\ttext\n')
    print(len(data))
    print(data[-1])
    for d in data:
        fileObject.write(d)

    print("---------- Object saved to '{}'".format(path))
    fileObject.close()


if __name__ == '__main__':
    train = load_data("./ComplexQ_data/train.txt")
    dev = load_data("./ComplexQ_data/dev.txt")
    test = load_data("./ComplexQ_data/test.txt")
    print(test[0])
    # save_data(
    #     r'C:\CIKE\diploma project\Git\Back\kg-search-backend\QASystemOnMedicalKG\data\dataset_ernie_tiny\complex_question_dataset_construct\ComplexQ_data\shuffled_data\train.txt',
    #     train)
    # save_data(
    #     r'C:\CIKE\diploma project\Git\Back\kg-search-backend\QASystemOnMedicalKG\data\dataset_ernie_tiny\complex_question_dataset_construct\ComplexQ_data\shuffled_data\dev.txt',
    #     dev)
    save_data(
        r'C:\CIKE\diploma project\Git\Back\kg-search-backend\QASystemOnMedicalKG\data\dataset_ernie_tiny\complex_question_dataset_construct\ComplexQ_data\shuffled_data\test.txt',
        test)
